{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF function 和 Auto Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0\n",
      "sys.version_info(major=3, minor=6, micro=10, releaselevel='final', serial=0)\n",
      "matplotlib 3.1.2\n",
      "numpy 1.18.1\n",
      "pandas 0.25.3\n",
      "sklearn 0.22.1\n",
      "tensorflow 2.0.0\n",
      "tensorflow_core.keras 2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "# 导入\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "print(tf.__version__)\n",
    "print(sys.version_info)\n",
    "for module in mpl,np,pd,sklearn,tf,keras:\n",
    "    print(module.__name__,module.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tf.function\n",
    "tf.function是可以把普通的python代码转化为tf中的实现\n",
    "\n",
    "autograp是tf.function所依赖的机制，这个机制使得python代码变为图结构"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(-0.95021296, shape=(), dtype=float32)\n",
      "tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "def scaled_elu(z, scale=1.0, alpha=1.0):\n",
    "    # z >= 0 ? scale * z : scale * alpha * tf.nn.elu(z)\n",
    "    is_positive = tf.greater_equal(z, 0.0) # 判断是否大于等于0\n",
    "    return scale * tf.where(is_positive, z, alpha * tf.nn.elu(z)) # tf.where实现三元表达\n",
    "\n",
    "print(scaled_elu(tf.constant(-3.)))\n",
    "print(scaled_elu(tf.constant([-3., -2.5])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(-0.95021296, shape=(), dtype=float32)\n",
      "tf.Tensor([-0.95021296 -0.917915  ], shape=(2,), dtype=float32)\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "scaled_elu_tf = tf.function(scaled_elu)\n",
    "print(scaled_elu_tf(tf.constant(-3.)))\n",
    "print(scaled_elu_tf(tf.constant([-3., -2.5])))\n",
    "\n",
    "print(scaled_elu_tf.python_function is scaled_elu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.8 ms ± 25.3 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n",
      "3.28 ms ± 32.3 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "# 转化后优势：快\n",
    "%timeit scaled_elu(tf.random.normal((1000,1000)))\n",
    "%timeit scaled_elu_tf(tf.random.normal((1000,1000)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### @tf.function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(1.9999981, shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# 使用@tf.function转化为图\n",
    "# 1 + 1/2 + 1/2^2 + ... + 1/2^n\n",
    "@tf.function\n",
    "def converge_to_2(n_iters):\n",
    "    total = tf.constant(0.)\n",
    "    increment = tf.constant(1.)\n",
    "    for _ in range(n_iters):\n",
    "        total += increment\n",
    "        increment /= 2.0\n",
    "    return total\n",
    "\n",
    "print(converge_to_2(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# python code 转化为 tf code，tf code转化为图结构\n",
    "def display_tf_code(func):\n",
    "    code = tf.autograph.to_code(func)\n",
    "    from IPython.display import display,Markdown\n",
    "    display(Markdown('```python\\n{}```'.format(code)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "```python\n",
       "def tf__scaled_elu(z, scale=None, alpha=None):\n",
       "  do_return = False\n",
       "  retval_ = ag__.UndefinedReturnValue()\n",
       "  with ag__.FunctionScope('scaled_elu', 'scaled_elu_scope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as scaled_elu_scope:\n",
       "    is_positive = ag__.converted_call(tf.greater_equal, scaled_elu_scope.callopts, (z, 0.0), None, scaled_elu_scope)\n",
       "    do_return = True\n",
       "    retval_ = scaled_elu_scope.mark_return_value(scale * ag__.converted_call(tf.where, scaled_elu_scope.callopts, (is_positive, z, alpha * ag__.converted_call(tf.nn.elu, scaled_elu_scope.callopts, (z,), None, scaled_elu_scope)), None, scaled_elu_scope))\n",
       "  do_return,\n",
       "  return ag__.retval(retval_)\n",
       "```"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_tf_code(scaled_elu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "```python\n",
       "def tf__converge_to_2(n_iters):\n",
       "  do_return = False\n",
       "  retval_ = ag__.UndefinedReturnValue()\n",
       "  with ag__.FunctionScope('converge_to_2', 'converge_to_2_scope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as converge_to_2_scope:\n",
       "    total = ag__.converted_call(tf.constant, converge_to_2_scope.callopts, (0.0,), None, converge_to_2_scope)\n",
       "    increment = ag__.converted_call(tf.constant, converge_to_2_scope.callopts, (1.0,), None, converge_to_2_scope)\n",
       "\n",
       "    def get_state():\n",
       "      return ()\n",
       "\n",
       "    def set_state(_):\n",
       "      pass\n",
       "\n",
       "    def loop_body(iterates, total, increment):\n",
       "      _ = iterates\n",
       "      total += increment\n",
       "      increment /= 2.0\n",
       "      return total, increment\n",
       "    total, increment = ag__.for_stmt(ag__.converted_call(range, converge_to_2_scope.callopts, (n_iters,), None, converge_to_2_scope), None, loop_body, get_state, set_state, (total, increment), ('total', 'increment'), ())\n",
       "    do_return = True\n",
       "    retval_ = converge_to_2_scope.mark_return_value(total)\n",
       "  do_return,\n",
       "  return ag__.retval(retval_)\n",
       "```"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_tf_code(converge_to_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(21.0, shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# tf.variable只能在tf.function函数外面定义，在里面定义会报错\n",
    "var = tf.Variable(0.)\n",
    "\n",
    "@tf.function\n",
    "def add_21():\n",
    "    return var.assign_add(21) # +=\n",
    "\n",
    "print(add_21())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([ 1.  8. 27.], shape=(3,), dtype=float32)\n",
      "tf.Tensor([ 1  8 27], shape=(3,), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "# 定义一个有输入的tf.function\n",
    "\n",
    "@tf.function\n",
    "def cube(z):\n",
    "    return tf.pow(z, 3)\n",
    "\n",
    "print(cube(tf.constant([1., 2., 3.])))\n",
    "print(cube(tf.constant([1, 2, 3])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 函数签名\n",
    "python为弱类型语言，为了类型更加明确，做类型限定\n",
    "\n",
    "添加了input_signature之后，函数就可以保存成tf支持的类型SaveModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python inputs incompatible with input_signature:\n",
      "  inputs: (\n",
      "    tf.Tensor([1. 2. 3.], shape=(3,), dtype=float32))\n",
      "  input_signature: (\n",
      "    TensorSpec(shape=(None,), dtype=tf.int32, name='x'))\n",
      "tf.Tensor([ 1  8 27], shape=(3,), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "@tf.function(input_signature=[tf.TensorSpec([None], tf.int32, name='x')])\n",
    "def cube(z):\n",
    "    return tf.pow(z, 3)\n",
    "\n",
    "try:\n",
    "    print(cube(tf.constant([1., 2., 3.])))\n",
    "except ValueError as ex:\n",
    "    print(ex)\n",
    "\n",
    "print(cube(tf.constant([1, 2, 3])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tensorflow.python.eager.function.ConcreteFunction object at 0x00000204285BF8D0>\n"
     ]
    }
   ],
   "source": [
    "# @tf.function py func -> tf graph\n",
    "# get_concrete_function -> add input signature -> SaveModel\n",
    "\n",
    "cube_func_int32 = cube.get_concrete_function(tf.TensorSpec([None], tf.int32))\n",
    "print(cube_func_int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "# 判断两个签名是否一样\n",
    "print(cube_func_int32 is cube.get_concrete_function(tf.TensorSpec([5], tf.int32)))\n",
    "print(cube_func_int32 is cube.get_concrete_function(tf.constant([1, 2, 3])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 图结构"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.func_graph.FuncGraph at 0x2042ba4c9b0>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ConcreteFunction object 有图定义\n",
    "cube_func_int32.graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'Pow/y' type=Const>,\n",
       " <tf.Operation 'Pow' type=Pow>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看func_graph图定义中的图操作\n",
    "cube_func_int32.graph.get_operations()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: \"Pow\"\n",
      "op: \"Pow\"\n",
      "input: \"x\"\n",
      "input: \"Pow/y\"\n",
      "attr {\n",
      "  key: \"T\"\n",
      "  value {\n",
      "    type: DT_INT32\n",
      "  }\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 查看图操作的定义\n",
    "pow_op = cube_func_int32.graph.get_operations()[2]\n",
    "print(pow_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<tf.Tensor 'x:0' shape=(None,) dtype=int32>, <tf.Tensor 'Pow/y:0' shape=() dtype=int32>]\n",
      "[<tf.Tensor 'Pow:0' shape=(None,) dtype=int32>]\n"
     ]
    }
   ],
   "source": [
    "# 取出图操作定义的属性\n",
    "print(list(pow_op.inputs))\n",
    "print(list(pow_op.outputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Operation 'x' type=Placeholder>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 通过名字获取operation或者Tensor\n",
    "cube_func_int32.graph.get_operation_by_name('x')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'x:0' shape=(None,) dtype=int32>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cube_func_int32.graph.get_tensor_by_name('x:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "node {\n",
       "  name: \"x\"\n",
       "  op: \"Placeholder\"\n",
       "  attr {\n",
       "    key: \"_user_specified_name\"\n",
       "    value {\n",
       "      s: \"x\"\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shape\"\n",
       "    value {\n",
       "      shape {\n",
       "        dim {\n",
       "          size: -1\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Pow/y\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 3\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Pow\"\n",
       "  op: \"Pow\"\n",
       "  input: \"x\"\n",
       "  input: \"Pow/y\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Identity\"\n",
       "  op: \"Identity\"\n",
       "  input: \"Pow\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "versions {\n",
       "  producer: 119\n",
       "}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 打印所有graph def\n",
    "cube_func_int32.graph.as_graph_def()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
